{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adadelta\n",
    "Adadelta 算是 Adagrad 法的延伸，它跟 RMSProp 一样，都是为了解决 Adagrad 中学习率不断减小的问题，RMSProp 是通过移动加权平均的方式，而 Adadelta 也是一种方法，有趣的是，它并不需要学习率这个参数。\n",
    "\n",
    "## Adadelta 法\n",
    "Adadelta 跟 RMSProp 一样，先使用移动平均来计算 s\n",
    "\n",
    "$$\n",
    "s = \\rho s + (1 - \\rho) g^2\n",
    "$$\n",
    "\n",
    "这里 $\\rho$ 和 RMSProp 中的 $\\alpha$ 都是移动平均系数，g 是参数的梯度，然后我们会计算需要更新的参数的变化量\n",
    "\n",
    "$$\n",
    "g' = \\frac{\\sqrt{\\Delta \\theta + \\epsilon}}{\\sqrt{s + \\epsilon}} g\n",
    "$$\n",
    "\n",
    "$\\Delta \\theta$ 初始为 0 张量，每一步做如下的指数加权移动平均更新\n",
    "\n",
    "$$\n",
    "\\Delta \\theta = \\rho \\Delta \\theta + (1 - \\rho) g'^2\n",
    "$$\n",
    "\n",
    "最后参数更新如下\n",
    "\n",
    "$$\n",
    "\\theta = \\theta - g'\n",
    "$$\n",
    "\n",
    "下面我们实现以下 Adadelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def adadelta(parameters, sqrs, deltas, rho):\n",
    "    eps = 1e-6\n",
    "    for param, sqr, delta in zip(parameters, sqrs, deltas):\n",
    "        sqr[:] = rho * sqr + (1 - rho) * param.grad.data ** 2\n",
    "        cur_delta = torch.sqrt(delta + eps) / torch.sqrt(sqr + eps) * param.grad.data\n",
    "        delta[:] = rho * delta + (1 - rho) * cur_delta ** 2\n",
    "        param.data = param.data - cur_delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torchvision.datasets import MNIST # 导入 pytorch 内置的 mnist 数据\n",
    "from torch.utils.data import DataLoader\n",
    "from torch import nn\n",
    "from torch.autograd import Variable\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "def data_tf(x):\n",
    "    x = np.array(x, dtype='float32') / 255\n",
    "    x = (x - 0.5) / 0.5 # 标准化，这个技巧之后会讲到\n",
    "    x = x.reshape((-1,)) # 拉平\n",
    "    x = torch.from_numpy(x)\n",
    "    return x\n",
    "\n",
    "train_set = MNIST('./data', train=True, transform=data_tf, download=True) # 载入数据集，申明定义的数据变换\n",
    "test_set = MNIST('./data', train=False, transform=data_tf, download=True)\n",
    "\n",
    "# 定义 loss 函数\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, Train Loss: 0.365601\n",
      "epoch: 1, Train Loss: 0.159966\n",
      "epoch: 2, Train Loss: 0.123347\n",
      "epoch: 3, Train Loss: 0.102201\n",
      "epoch: 4, Train Loss: 0.087986\n",
      "使用时间: 59.26491 s\n"
     ]
    }
   ],
   "source": [
    "train_data = DataLoader(train_set, batch_size=64, shuffle=True)\n",
    "# 使用 Sequential 定义 3 层神经网络\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(784, 200),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(200, 10),\n",
    ")\n",
    "\n",
    "# 初始化梯度平方项和 delta 项\n",
    "sqrs = []\n",
    "deltas = []\n",
    "for param in net.parameters():\n",
    "    sqrs.append(torch.zeros_like(param.data))\n",
    "    deltas.append(torch.zeros_like(param.data))\n",
    "\n",
    "# 开始训练\n",
    "losses = []\n",
    "idx = 0\n",
    "start = time.time() # 记时开始\n",
    "for e in range(5):\n",
    "    train_loss = 0\n",
    "    for im, label in train_data:\n",
    "        im = Variable(im)\n",
    "        label = Variable(label)\n",
    "        # 前向传播\n",
    "        out = net(im)\n",
    "        loss = criterion(out, label)\n",
    "        # 反向传播\n",
    "        net.zero_grad()\n",
    "        loss.backward()\n",
    "        adadelta(net.parameters(), sqrs, deltas, 0.9) # rho 设置为 0.9\n",
    "        # 记录误差\n",
    "        train_loss += loss.data[0]\n",
    "        if idx % 30 == 0:\n",
    "            losses.append(loss.data[0])\n",
    "        idx += 1\n",
    "    print('epoch: {}, Train Loss: {:.6f}'\n",
    "          .format(e, train_loss / len(train_data)))\n",
    "end = time.time() # 计时结束\n",
    "print('使用时间: {:.5f} s'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x103f3a5f8>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXeYJFd97/09lTr35Dybg7RBq7QSCiBLCEmAEfjF8JJM\nMDzWy8vFYBsbg7Hxq3vtF78XbF8Trm1xjSVssNAFLCGiSEIEIWklrcKuwq42zmyYPNO5q6rr/aPq\nnDrVXZ2me6Z7es7nefRI6umpPl3TfX7n9/0lYlkWBAKBQLD+kFq9AIFAIBC0BmEABAKBYJ0iDIBA\nIBCsU4QBEAgEgnWKMAACgUCwThEGQCAQCNYpwgAIBALBOkUYAIFAIFinCAMgEAgE6xSl1QuoRH9/\nv7V58+ZWL0MgEAjWDI8//viMZVkDtTy3rQ3A5s2bceDAgVYvQyAQCNYMhJCTtT5XSEACgUCwTmlL\nA0AIuZUQcsfi4mKrlyIQCAQdS1saAMuy7rcs67aurq5WL0UgEAg6lraOAQgEgs5D13VMTEwgm822\neilrmmAwiPHxcaiquuxrCAMgEAhWlYmJCcRiMWzevBmEkFYvZ01iWRZmZ2cxMTGBLVu2LPs6bSkB\nCQSCziWbzaKvr09s/g1ACEFfX1/DXpQwAAKBYNURm3/jNOMedqQBuPOXx/Htp8+0ehkCgUDQ1nSk\nAfiPR0/j/qeEARAIBLURjUabfs277roLO3bswI4dO3DXXXf5PufkyZO48cYbsW/fPlx//fWYmJhg\nP/vTP/1T7N27F3v37sXXvva1pq8P6FAD0BNRMZfKt3oZAoFgDWBZFgqFQlOvOTc3h9tvvx2PPPII\nHn30Udx+++2Yn58ved4f//Ef413veheefvppfPKTn8THP/5xAMB3vvMdPPHEEzh48CAeeeQRfOYz\nn8HS0lJT1wh0qAHojWjCAAgEgrKcOHECF1xwAd71rndh7969yGQy+MQnPoGLL74YV111Fc6fP8+e\n98pXvhL79u3DjTfeiFOnTtV0/R/84Ae46aab0Nvbi56eHtx00034/ve/X/K8w4cP45WvfCUA4IYb\nbsB9993HHr/uuuugKAoikQj27dvn+/uN0pFpoL0RDfNpvdXLEAgEVbj9/kM4fKa5J9vdo3H85a17\nqj7vyJEjuOuuu3DVVVeBEIKrrroKf/3Xf42PfvSj+OIXv4g///M/x+///u/j3e9+N9797nfjS1/6\nEj70oQ/h3nvvxVe+8hV8+tOfLrnm9u3b8fWvfx2Tk5PYsGEDe3x8fByTk5Mlz7/44ovxzW9+Ex/+\n8Ifxn//5n0gkEpidncXFF1+M22+/HR/5yEeQTqfx05/+FLt3727sxvjQmQYgrGE+nYdZsCBLIttA\nIBCUsmnTJlx11VUAAE3T8LrXvQ4AcPnll+OHP/whAODhhx/GN7/5TQDAO9/5Tnz0ox8FALzjHe/A\nO97xjobX8JnPfAYf/OAHceedd+K6667D2NgYZFnGzTffjMceewzXXHMNBgYGcPXVV0OW5YZfr5iO\nNAA9EQ2WBSxmdPRGtFYvRyAQlKGWk/pKEYlE2H+rqsrSKmVZhmEYFX+3mgcwNjaGBx98kD0+MTGB\n66+/vuT5o6OjzMAkk0l84xvfQHd3NwDgE5/4BD7xiU8AAN7+9rdj586ddb2/WmjLGECjzeDopj+X\nyjVzWQKBYJ1xzTXX4O677wZgb/qveMUrANgewMGDB0v++frXvw4AuOWWW/DAAw9gfn4e8/PzeOCB\nB3DLLbeUXH9mZoYFoD/1qU/hve99LwDANE3Mzs4CAJ5++mk8/fTTuPnmm5v+/trSA7As634A9+/f\nv//3lvP7rgEQcQCBQLB8Pve5z+F3f/d38elPfxoDAwP413/915p+r7e3F3/xF3+BK664AgDwyU9+\nEr29vey/9+/fj9e//vV48MEH8fGPfxyEEFx33XX4whe+AMDul0SNTTwex7//+79DUZq/XRPLspp+\n0Waxf/9+azkDYQ6dWcRvfvYX+KffuRyv3ju8AisTCATL5bnnnsOuXbtavYyOwO9eEkIetyxrfy2/\n35YSUKO4HoBIBRUIBIJydKQB6AnbBmA+LQyAQCAQlKMjDUBQlRHRZOEBCARtSjtLz2uFZtzDjjQA\ngJ0KKgyAQNB+BINBzM7OCiPQAHQeQDAYbOg6bZkF1AxEOwiBoD0ZHx/HxMQEpqenW72UNQ2dCNYI\nHW0AZpPCAAgE7Yaqqg1NsRI0j46VgHrDwgMQCASCSnSsAeiJaCILSCAQCCrQsQagN6IhnTeR1c1W\nL0UgEAjako42AIAoBhMIBIJyCAMgEAgE6xRhAAQCgWCd0rEGQLSDEAgEgsp0rAHoczwAUQsgEAgE\n/nSsAYiHVEhEeAACgUBQjo41ALJE0BPWcG4x2+qlCAQCQVuyaq0gCCERAP8TQB7Ag5ZlfWWlX/Nl\nW3vxo+fOI28UoCkda+sEAoFgWTS0KxJCvkQImSKEPFv0+KsJIS8QQo4SQj7mPPxGAF+3LOv3ALy+\nkdetlTfv34D5tI4fPXd+NV5OIBAI1hSNHovvBPBq/gFCiAzgCwBeA2A3gLcRQnYDGAdw2nnaqpTn\nXrdjACNdQXztsdPVnywQCATrjIYMgGVZDwGYK3r4SgBHLcs6ZllWHsDdAN4AYAK2EWj4dWtFlgje\ndPk4HjoyjTMLmdV4SYFAIFgzrMRGPAb3pA/YG/8YgG8C+G1CyD8CuL/cLxNCbiOEHCCEHGhGv/A3\nX74BlgXce3Cy4WsJBAJBJ7FqQWDLslIAfreG590B4A4A2L9/f8Mjgzb2hTHSFcRLU6lGLyUQCAQd\nxUp4AJMANnD/P+481jL6owHMJHOtXIJAIBC0HSthAB4DsIMQsoUQogF4K4Bv1XMBQsithJA7FhcX\nm7Kg/qgmDIBAIBAU0Wga6H8AeBjABYSQCULI+yzLMgB8EMAPADwH4B7Lsg7Vc13Lsu63LOu2rq6u\nRpbHEB6AQCAQlNJQDMCyrLeVefy7AL7byLWbSX8sgNlkHoWCBUkirV6OQCAQtAXrojy2PxqAUbCw\nmNFbvRSBQCBoG9rSAKxEDACAkIEEAoGAoy0NQLNjAAPRAABgRrSGFggEAkZbGoBm0x+jBkB4AAKB\nQEBZHwYgKgyAQCAQFNOWBqDZMYDukApZIsIACAQCAUdbGoBmxwAkiaAvomEmIWIAAoFAQGlLA7AS\niGIwgUAg8LJ+DEBMGACBQCDgWT8GIKqJNFCBQCDgaEsD0OwgMGDXAkwnc7CshjtMCwQCQUfQlgag\n2UFgwI4B5I0CEjmjadcUCASCtUxbGoCVoD/mtINI2HEAy7Lw1UdOYTpROS5gmAU8/NLsiq9PIBAI\nVpv1YwCK2kEcPruEP/vPZ/APP36x4u/dd/AM3vbFX+PXx4QREAgEncU6NAD2if9XR+0N/d4nzyBV\nQRZ69Lg98/5bT51Z4RUKBALB6rJuDcAvX5pBRJORzBkVN/fHT80DAL7/7DnoZmHlFyoQCASrRFsa\ngJXIAuqNaFBlghfPJ5A3Cnj0+BzeeNk4LhiK4auPnPL9nYV0HkenkrhkQzfmUnn8SsQCBAJBB9GW\nBmAlsoBkieDWi0fxjccn8dMXppDOm7h2ex/e/rKNeGZyEc9MlBqbJ5zT/x/etBOxgIJvCxlIIBB0\nEG1pAFaKD1y/DVnDxCf+8xkQAly1tQ+3XjwKwJaEijlwYh6KRHDl5l7cvGcY3z90DjnDXO1lCwQC\nwYqwrgzA9sEYbt49hJlkHntHu9Ad1tAb0TDaFcThM0slz3/85Dz2jMYR0mS8/pJRJLIGfnR4qgUr\nFwgEguazrgwAAHzg+u0AgGu397PHdo3E8dxZrwHQzQKemljAZZt6AAAv396P0a4g7jlwevUWKxAI\nBCvIujMAF2/oxpffeyXe/xtb2WO7R+M4NpNCVnflnefOLiGrF3C5YwBkieBNl4/joSPTOLOQWfV1\nCwQCQbNZdwYAAK7bOYDusMb+f9dIHGbBwovnE+yxyXl7k9/aH2WPvenyDbAs4BuPT6zeYgUCgWCF\nWJcGoJhdI3EA8MhA82kdANATUdljG/vCuGZbH+55/DQKBdFUTiAQrG3a0gCsRB1AJTb1hhHWZDx3\n1vUAFjJ2y4gezlMAgN+6ZAyn5zI4Op2seM1kzsBcSrSfFggE7UtbGoCVqAOohCQRXDgcw2HOA1hM\n6wgoEoKq7Hnuy7b2AgAeOzFX8ZqfvPdZvOdfH23+YgUCgaBJtKUBaAW7R+1MIDovYCGtozusljxv\nY28YA7EADpyYr3i9w2eXcPjMEvKGaB8hEAjaE2EAHHaNxJHIGphwgr/z6XyJ/AMAhBBcsbmnxAOw\nLIv1CrIsCydmUzAK9r8F64N7n5zET58XdSKCtYMwAA40EPzCOTsOsJDR0RUq9QAAYP+mXkzMZ3B2\n0U0H/cJPj+LV/+MhAMBUIoesbhsDPrNI0Nn8089ewp2/OtHqZQgENSMMgMNYdwgAcD6RBWDHAPwk\nIAC4cguNA7gy0DOTi3hpOoWpRBYnZtxT/5HzlYPFgs4hbxaQyYtWIZ3MbDKHRFZv9TKahjAADr0R\nDYSATQibT+fRHSqVgADgwuEYIpqMA5wMdG7RNhzPn03g5GwaABBSZRyZah8P4PGT87j7Uf/Op4LG\n0c0CMnprDcADh87hn3/2UkvX0Mm8764D+NT3nm/1MpqGMAAOqiyhN6xhKmEPjl/I6OiO+HsAiizh\nsk09Hg/gDDUA55Zwci4FRSK4amsvXlwFD8AsWHj+XGkvo2K+8uuT+OS3Dom5BiuEblhI51s7c/rr\nj0/gLiFDrRjTiRxmk5XHyK4lhAHgGIgFMO3o93mjUNYDAIDLNvbg+XNLyORN5I0CGzTz/NkETsym\nMd4Twq6ROE7MpBrKBJqYT1ctOnvg0Dm85h9+7olJ+LGQ0ZE3Cjg23frA9H0HJ3HDZx6E2UEFde0g\nAc2n8y3zQh58YQrX/s1PPC1VOo2sbsIwO+cz25YGYLULwSjUAMyn7QKucjEAANg+GIVlASfnUji/\nlIWTPYrnziVwcjaFTX0R7ByKNZQJNJvM4YbPPIjvPXuu4vPOLNqvX23A/WLG1i6LG9+1gqdOL+L4\nTApLmc7RU3WjgHSLN7+5VB7pFhmho1NJTC5ksNRBGnkxWd2E3kGHlrY0AKtdCEahBmCBtoGoYAA2\n90UAACdm0ji3ZMs/u0fiODqVwPHpFDb3hbF90O4jtNxMoKlEDrppYWI+XfF5C47BSmYryw/UABxu\nAwMwm7KN1WIHGYD28AB05IxCS1qV5B1pUe+gEzKPZVnIGgUYHSShtqUBaBWuAbA31K4KEtCm/jAA\n4MRsinUHfeWFg9BNC6m8iU19EWwfjEIiy88ESjgberVNkhqsRIXh9vx12sEDmE3a93ihQwyAZVnI\nmwXkjELLZC2zYLHPbrYFg4t0w37fnbRB8uimBbNgCQmoUxmIBpA3Czg5Z5+4K0lA8aCKvoiGk7Mp\nlgF0w4UD7Oeb+sIIqjI29oYrZgJ96nvP4UeHz/v+jKabVTUAzs8TFTwAy7JcD+CMW/HcKmjMhG5Y\nq4lhFnDPgdNN3ajNgsVkwFZp8EsZHfQttcIT0TvcA6BGVS90joETBoBjIBYA4J7Y/SqBeTb1hXF8\nJoWzi1nEAgr2jXdDkyXnZ7ZEtHUgiuMz/hKOWbDwLz8/jnsPTvr+fKlWA8AkIPt5Pz8yjf/znx/2\nbAI0sD0YC2A2la8aL1hpZhwPoBUS0K+PzeGjX38aT56q3M6jHvhNr1WZQHOcMW2FEXINQOdskDw0\nuC08gA5lMBYEAHZir+QBAMDm/ghOzqZxdjGD4a4gVFnC9sEoCAE29NqFZUPxIM47MYJiphJZGAUL\nk2UGzNQtATnPf+zEPB49PodvPeUaFnqNq7f1AQAOtVAGKhQszLUwBpB0pLJkFcmsHvhMr2y+NRvg\nPNd9thUeAI0BdNIGyZPTO8/ACQPAwXsAfp1Ai9ncF8HZxSyOz6Qw4lQS79/cgwuGYggo9u8Ox4OY\nS+V9h8nToTP038XQDb1apgzNWqIbGn3+lx8+yaQeutFetdU2AH4zkFeLBU6qoMarXp46vVDWcFaD\nnuSama6Y5zaFtN4iDyDVHh5AvoM2SB7mATQ5xjO5kMH/861DLYkdCQPAQQ3AuaVsVfkHsD0AAHjx\nfBIjcdt7+LPX7sI977+aPWe4y77m1FKp5EI3sKlEztdA0I282il50dlEl4o8hkNnlvDk6QXPYxt6\nwhjvCbU0EMwX0izXAHzgK0/g099fXkUm3RybuUnyp8JWpWHOp1vrAXR6EJj292r2+/vJc+dx569O\nVM32WwmEAeCIBxVoin1Lqsk/ALC5L8z+e6TbNgBBVUY86P7ukGMY/GQg/gR7ZqH058Ubuh+6WWDZ\nP9QDWEjnsaU/gmhAwb89fNJzja6QiguH4y1tUjfNGYDlSkCzqRwL1tcLPcllmijV8AagVamgcyn3\nXrY2BtCZEhALAjf5/dHvQCtaxwsDwEEIwaDjBZTrBMpDA70AMNIV9H3OsPP4OR8DwA+X95OBaBbQ\nUtYom7XDb6B81tBodxC/dekovvPMWRhmwWMA+qPask/elKxu4nWf+zkePV55MI4fNAVUUyQsZurP\nAsrqJrJ6oax0Vo2V8AD4L2+rDADvAbSiGpfVAXRQlgyPKwE19/3R72ZOGIDWQ2WgWiSgrpCK3oj9\nvJGukO9zhh0PgKaK8kzOZ5inMblQepqlHoBZsMoGLPmNPMl5DF0hFbtG4sgbBUwn+doGFdGAUlMA\nNJM3ywa8JubTeHZyCc9M1l+tTSWgrf2RZRkiKo2Vk86qkc1TD6CJQWBPDKBVHoBrAMrJUN9++gwe\nfml2RV5f7/AgsCsBNff9LWXsz2ErYifCABQxELUNQC0SEODKQOU8gK6QioAilZWALtvYA4lU9gCA\n8lIJ3dhDqsw29cWMga6QytZ0ZiGLpYwOQoBYUEE0qCCdN6sGnX77H3+Fv33gRd+f0TTO9DIyaWZT\neUjEnq62HAmILx476yOdVSPrnLSaGwNw72UzDUs9zKfy6I/aB5Jy7+3vHngR//0HK9PNknpBnZQl\nw0M9gGa/PyEBtRHUA+iq2QDYMhDNAiqGEILhriDOFQWBLcvC5HwGG3vDGI4HMeGT0ZLIGlBlAqB8\nsJQ+vqE3hIQjFS1ldMRDKvNKzi1msZjREQ+qkCSCaEABUD0NcmI+jSdO+ufKUxlnOafdmWQOvZEA\neiPasiqBeaOxnEygTL75MQD+y9uqIPBcOs/mWpSTodJ5E89MLK6ITEWNYOcbABEDWFFa1QwOcGsB\nKnUC5XnV7iHceOEg21T9GIoHcb5IAlrKGEjlTYz3hDDWE/L1AJYyOkadL3S5VFCq+27oCSOR1e2C\nL7Pg8QDOLmaYLATYXgBQ3QBk9QJemvZvY0F7+SzHA5hJ2ifVrrCKxbRed1UybwyXEwdY6Swgv+ta\nloU/uPtJ/PLoTNNes5j5VJ59XsrFADK6CaNg4cnTzSuCo7S6F9BKN6GjnuNKxQCoATg6lcD3nNjd\nStOWBqBVzeAAPgZQmwfw2otG8C/vuaLic4biQTZpjDLhaP5j3SGMdYfYLGKeRNbAeI/9hS4nlbD0\nzt4wkjkDCxlX6+8KqQiqEvMAqAGIBux/V2oeZxbs3jazqbynwIhCJaDUMk6Ss8kc+qMBdIVU5M0C\n01Z5fvzcedzz2Gnf3+fvhZ/nVI3MCtcB+J2uzy/lcO/BM3joyHTTXrOYuVQeg7EAFImUNW50bcsJ\n3lfDjQGs/kn25GwKl/7XH+IpJ+15JchxHkAzW6lQw0WDwN9/9hz+7688gdUwo21pAFoJNQC1xgBq\nYTgewLnFrOdDQ9M+R7ttD+DcUtbzxaHTpTb02DGG8jEAHbJky0wFCzjreBpdIRWEEIx2hXC22AAw\nD6D8iYnfHI/6eAE0kLscKWE2lUdfVGNe1kJRJtAvj87g//q3x/HRbzyNLz50rOT36b2IaPKyPIAc\nSwNdmSwgPwmIelKV+jU1gm4WsJQ10BPREFJl3zUYZoEZqsdOeA1AOm80XIjUylYQx6ZTMAsW+/yv\nBPx3oplFW8wDMO3r54wCJAIoEmnaa5RDGIAi9ozGsbkvjN0jzfM+huJB5IyCV7t2ij7GekIY7wnD\nLFg4z/XnoRvFht7KBsAeXakyWYd6EnSzH+4K4uxixjPknspVlTYj/gT50pSfAaAewDIkoEQOfZEA\nM7K8pHN0KoH3//vj2DoQwWsvGsZff/c53HPA6wkspvMgBLhgOOabPcXz4vlEyYbUCgnomGMAVmr+\nAb2HfRENQU329W6ohKHKBE+cXGBrLhQsXP/pB/FvD59oaA20EKwVEhCtLVlJ48N7qs2qBjYLFvse\n0kNEziggoMggRBiAVWe0O4QH/+QGbOSKvBrFrxZgciGDgCKhL6KxwB1/mqUZQEPxIGSJlPcAMjq6\nwipiTvEZrSbkDcC5RTsLiAa24zXEAPjT8VE/A0BjAHWeojN5E6m86XgA9nr49/aFn74EAuBL77kC\n/+Mtl+KisS7c+csTnmvQgPaG3nDFIPCDL0zh5r9/CN995qzve1sxA+DrAdhDgVbKA6CxoJ6IhrAm\nszUspnUm4dHH9m/qRUY3cchpB3J2KYupRG7ZrTUoTAJqQR3ATJ0G4MlT87j9/kN1STm8UW2WoeFl\nWGYAdBMBdXW2ZmEAVgG/WoAzC1mMdYdACMGYo/Pzp1m6UcSDCrpCasU00J6whphzqp8s8gBGuoI4\nn8j5SkCVNqPqEpCTBVSnB0ANx0A0gHio1AM4NZfGntEujPeEoSkS9o13lRTRUW9mrDuEswtZX3c8\nq5v4i/ueBYCS+ErGOck1NQbgfHljAcX3nrgS0Mp4ALQGoDdsS0DUuP3J15/CH91zEIBrAK7babct\nf/S4XQ9wcsY2To0WIrUyCEw/j7Xm6P/w8Hn86y9P4FQd1eT8gaFZtQD895re/6xeQEARBqBj8GsH\nMbGQYRs/9QBOz7kbFZUKYkG1igHQPRIQPcVRfX2kKwSzYEE3rRIJqFIQmH7YowHFNxOInrjSufo2\nUfpF7YtqTALiq4En5937Yq/fbqbHb9aLGR3dYRVjPSEYBQtTiVLd93M/OYLTcxnIEmFrpZSLAViW\ntey4QN7ZEOIh1T8G4HhRSyvlAaRcDyCoyszITcxnWAoy/Ztu7A1jY28YB52A6YlZexNs1CC2MgZA\n/8a1FlPRe/F4mTRnP3gJqFnVzn4GIGeYrJnkSiMMwCrgGgD7Q5rI6njh3BK2DdgjI4OqjG0DERzg\nPox0o4iHFMSrGYCwxk71E/MZVvAFeAvUqAGIaI4HUIMEtGc0jon5jGdzyBsFtr56JSDqAfRFA+h2\nqq2pB5A3CjifyLJURnv9bi0D/56pBwCUpoImcwbueOgY/o9Lx7ChJ8Qylth7KxMD+MGhc9jzl9/H\np3/wvG9O9v8+cBp/9e3Dvu9Ld57fFVJLNtJUzsAZZ/0r5gE4ElCvEwSmxWiLGZ15JPT9hjUZFw7H\n8MI5ux/UydnmeACtrAOoVwKin+8D9RgAozEPYCmr4w/uftIzBIlPXfXGAIQH0DFojtZPpYxvP30W\nWb2A37p0jD3n+gsG8etjs+yDSTeKuOMB8MFDs2DhgUPn2AjAbi4GMDmfQSygQHIyCIZ9DAAtBqvF\nA9g71gXLgscLoHJDQJHqDgLPJBwPIKIhosme+Mb5JXu4/Xi31wMA4MnuWHIkoHEmnXkNwNmFDHTT\nwvUXDKAvGsBM0fCbcmmgL02nULDsOMSb/+lXns1kYj6Nv7jvWXz51yd95+3S53b5eADHHYllvCfE\nyv6bzZxj5LrDKkKaKwEtpPNI5bweT1C1DcCJ2TSyuokT1AD4pOPWA93AWtEKgnqWtRoA+jcqV+jo\nR65BCejJUwu49+AZPHnKTVXlD3bUe8kZBRED6DSG4kE8f9YexXjPgdPYORTFxeNuptH1FwwgbxTw\n8DG7UIiesGM+MYCHXpzGbf/2OL76yEmk8ia6nf4+gL258VXMo1yPIr7Bnd0PyL7m3z3wAv72gRc8\n66Wb40Vj9hppEBNwT1sbesPI5M26Amk05bMnooEQgu6QyqqBqVbPS0DDXDGbew3bAFBPoVjjp9PO\nBmNB9Ec15nVQ3Epg70a9kM4jpMr4mzdehKcmFvFDblTnf73/MJuq5tfYj25+3WG15LrUeF6yoRsZ\nvXx/pUaYS+cR0WQEFNk2AE4fp1TeRCpHPQD73yFNxgXDcZgFC0enkjjpSEDL6avEQzewVvS0cT2A\n2j6L1AC8cD5RcwFZoxIQlemK5UxKXkhAncub94/jiVML+MS9z+LJUwt48+UbPGleV27pRUiV8eAL\ndqEQ9QCiAQVdIcXzQXnBaeX82Z8cBQB0RzRPJTK/0XeHVeZOegxA0G0I98Dh8/jO00WZMs6HdNdI\nHBLxZgLNpmj1sa3B1/OFp6fRsDNsh1YDA+5J3k8Coh4AnW3cHVYR1hT0RwM4MeMaJ8BuEgfYNR39\n0YBHAioULCZ1ZHSv8aKB8jfv34DRriC++sgpAMBPnj+PBw6fxyt29AMAOzHz6GYBhNh/r2IP4KWp\nJCQC7HMM/kpkAk0t5TDoSI0hVUZWd9OOM7rd94m2vghrMi4YtuXHF84lXA+gAQnIsqymN4M7OpXw\nLUKkTCdysCx7UDv1Sms1rlndhCIRWBY8J/Jqv0NZznuka+TvM/XsA4rEZQEJCajjeM81m3HLniF8\n9ZFTUCTikX8AIKDIuGZbHx58YRqWZecGRzQZiizZEhDXEppuxvSk2x1SIUsEEc3ZVLmNnhDCZJRi\nD4BuRDPJPCbmM55sGnraoSftU9ymR4vANjo1CvUEgtN5A2FNZhIV793Q9th83CKkyegOq8wDSObs\ngiX6XnaPxlk6I4V5APEA+qIBzKfzrMiOfvliQQUFy3tapQZAlgjeeuVG/OLoDB49bs8P3jkUxe2v\n3wMA7MTMkzctqLJkp2AWS0szKWzsDaMvYhcZ1hMHyOomvv30Gd9AN8/ZxQy7bzQLiD80ZHSTrSuk\nytjcF4FGmkKaAAAgAElEQVSmSPj5kWn2t27EAJgFC9SWNisN9G1ffASf/ckR358tpnVc+//9BPc/\nfRbz6TybMFe7BGTgovEuSKT2QDAfA1iOF0dTdYs9AFki6AlrIgbQyRBC8Ok3X4wdg1G8/uJRVnHM\nc/2Fgzg1l8axmRSWMjrT9btDmqcl9NGpJK7c0svmDtPW1TQQXDzLgJ6ieWko5ngApjOfN296pQ0q\nY4RUGRt6wp50Oaq3bnQa4dXTEC6VNxHWXG/FloDs603OZ9AfDZSM4hyOB1kQmG5qNMtp76g93IaX\nL6YSWQQUCbGAgoGoBstyg6R0E6RtvPn5vTS4DABvuWIDZIngnf/yCJI5A59/+2XY1BeBKhN/A2AU\noMkSQppSKgFNJbFtIMrSXuuJA/zw8Hl88KtP4qr/98d4752P+bYVB2wPicplIU1GOm940mvTOYMF\nhoOqfbDYPhDFj5+fAmCfQBvJAuKll7zRuAeQzhuYTuSYMS9mKasjbxRw8NSCJ8urHgmoPxrArpE4\nHj9ZW1uMrF5AyPlsLqcQrJwB6Aqp0BSJfYaFBNShxIMqvv8H1+G/v2mf78+vd/Kzf/r8FBJZA/GQ\nd0NfzNiN045OJXHhcAy3XbcNgDt2khqMUgMQhESAKLfx0iDwXMo9PZ3iNja6UQY1CRt7wzjFpajO\npHLQFIkNz6mnIVw6ZyAScD/c3WF3OM2ZRW8KKGW0O8QkIGoA6GZ60VgXjILFMloAWwIajAdACEGf\n096bBp/pl48azUzxl9ExkkPxIF61axA5o4D/9oa92DkUgywRbOgNs6wZHt0sQJUJwpqMvFnwtPU4\nOZvGpr4Iy8yqxwOgssG7rt6MR47N4tbP/wJPnvKeWA2zgPNLWRbvCToSEJ9tkspzHoDjKV4wHGNe\n4I6haEMeAO9JNcMDoK1SyhUr0g346HSS/W2B2jtqZnQTYU3GJRu68cxEbU0ns7rJDlnL6Xc070xs\ny/ISUNZgBkAEgdcBskSgyP63fUNvGDuHovjRc+eRyLkeAF8wdX4ph2TOwPbBKH7nZRtx/wdfju2D\nMQBufn+8yADcuGsIt148ymQX+txkzvCcsE7NuRtbVjchEUCTJWzoDWEmmWMn25lEHv0RjW3k9aSC\nFnsAXSEVC05H0Mn5jCcDiGK3s3AMgGMsaA3BXidI/eykKwNNJ3Ksq2u/YwBoIDjDDIDq+X/AzS6i\n3P76vfj82y/Fm/dvYI9t6g2zvHke3SxAU2wJCHC9oqwjvfRFNTYqtJ6ulVQj/rPX7sI3P3AtgqqE\n3/lfj7DALmC3QShY7lhSuobzXAvyVM5gMQB6ir1g2P7cKBLBlv5oQ0FgXhJpRpCbSn7l4iV0Az56\nPuEJ8tdqfNJ52wB0h/3rNvzI6gVWcLmcYre5MkHgeFCBJntjAEHhAaxPbt49jMdOzOPUXJqdGLuY\ndKAz/X/7QBSEEFzEZRIVP5/ym/tG8A9vvdTzWDRoewC8+8xLG5m8iZBq9yOh/YhOO20mZlM59EUD\nCKn269WTCprOGyxWAQC7RmJI5gw8M7mIyYUMRrtLB+uMcsVg/GhLwE6tjAcVPHvGPcVNJXJssA8d\nkDJT1Lyux5GAeLlmocgADHcF8bp9o561bOqL4ORsqiTzKW8UoMoSk6/o1DHq3fSENfb3qacYbCmr\nI6TK0BQJFwzH8Gev2YVU3mSppYB7WuZjAABwjsucSjsegKZIkJ2DADUAG3rDiGhyQ2mgXgNQ3+Zo\nWaXdNemgn3LeEn2NM4tZ9rkNqTLrR1SNbN5ESFUQUGQYBaum5m453gNYThYQk4C8cad4SEVAlbyF\nYMIDWJ/ctHsIZsHC6bkMOzHS0+5UIoejU7bUsX0wWvK7dIOpZZZBLKAgmTdYxowmSx6dP6ObTCpg\nBsD5+WzS7uZJPQC/6tnFtI6//+GLJafKVM5EmMtYumXPMBSJ4MsPn0TOKLDiLp5hrhiMpozSe0II\nwd6xLhziRlNOLWUxGLcNAJWAaNyCnr56iyQg3Swg7aTUVmJzXxjpvFlSXJY37RgA8wCce0K/9N1h\n1fUAnPdwei5ddeNZyrhSIABsGbDjLrwBoHEBGuthBoCL6aTydgwgxMVXLhiyDcCmvjCCqtyQBMRv\nvPXKIx+++yA+cs9TnsfO0KB/GWPJ37dHjs9CkyX0RbWavA/LspDWTYQ0CZoTbK1FOsoaJvOyG8kC\nyvp4nR4PQASB1y8XjXVhKE41ffvDtn0wisFYAPcenMSRqSTiQcU3iBwL+McA/IgFVVgWWHbPReNd\nJQaAnmZpts8pZgDsbp50s/ObCfA/f3YU//DjI3jkWGnbYd4D6A5rePmOftx3cBKANwWUwheDFXsA\ngH3Pnjtnd/3M6iaWsgbzAKh7TbtF0tMX9QDol5Fdt0ob8E399gZcHAcokYCKPIDusOrpwTSTzOGV\nf/sgvv30mYqvt5TVmeEA3Al0vAGgcgmLAThr4Ivn0jmT6d6Uka4gRrqC2D0SR4ALQi6HfAMewIvn\nE3ikaD7BOVY57W8A+Dz8x0/Ooy+q2ZtoDQYgbxZgFiyENYVttNXeO22nEmUSUH1GzrIsdhjgX2uJ\nCwJ7YgBCAlqfSBLBq3YNAXCDuqos4W1XbsTPXpzGL47OYPtg1LdVbLksID/oc4/NpBBSZewaiXkk\noCxnAPqc9gKn5zKwLAszzuxZquUXB4GXsjq++ms7h/5kUbOtVM4bAwCA1+0bZZuGXxB4hHVTzWAh\nrdvZNtxJds9YF/JGAUfOJz0poACcQLDGAoWZ4iBw0UZd7d5tcoxhcRyASkAhjRbkGc51HQ8gpEF2\nKrATWQMnZlLQTasko2g+lcd773yMpdouZXVPTCeoyhjrDpVIQCFVZp4CvTdnF7NM7knlDGS4LBZ6\nb77zoVfgQzfucLKACssedNJIDCCRNTC5kPE00aOtM5J5w7fymj+BZ/UC+qIaVFmq6bX5DDe60Vbz\nfuhBwZWA6rtPyZzBPuP0EEJrWuLO3PCcbicPmAWr8zwAQshWQsi/EEK+vlqvuVa5ec8wAHhc/7e/\nbCMkYqcg+sk/gBsErskAOM89PpNCf0zDpt4IFjM6C7LyKW+EECcTKI1zS1nkjQIG40HWU6g4iPbv\nvz6JRM6AROCpH7Cf680CAmzZS3MC4+PdpW24h7nh9vQLwxvAvaNxAMCzZxbZSZ8GgQE7EFwcBO6N\neIPAfp6FH+M9YUjEzwOwoMqE3TMacKWSVU/EbcW9lNVZ9fJsUaO6w2eX8JPnp1hx0lLGYO27KVv6\nIzjGS0BLGYx0B9k9YUHgxSzrRGtLQGZJim2v0zwu4DxOM5g+8JXH8exk7SNZeQmlfgNg36NjXLX5\nWacmxLL8Y0zFMlN/NABVITVJM3xPpFolIGoAYsv0AGgGEH+tdN4ez8l7ANQQtVUMgBDyJULIFCHk\n2aLHX00IeYEQcpQQ8rFK17As65hlWe9rZLHrhau39uF1+0Zw7bZ+9thQPIhb9tieQTkDUC4I7Ac9\nyZyYSWEgGmA6P5V5aBCYsqE3hNNzafz4OTtv/Lod/SxGwJ/csrqJL/3iBF6xox/bBqIlJ+XiLCC6\n3ut2DiAWVDxGjxLW7HYY9mjLfMm0ts1OiuWTp+YxteRWAVP6oxoLAtPgbHEa6FKNBkBTJIz1hEo9\ngBIJyL4n85wHANheXSKrs7kNM0WVrnQjonULxR4AYBuA49NJdlo/s5D1tPygm3wiZ7Cguh0ENtjf\nrBhXCilgOpnDd585h5+9WPv4SrohKhKp63RsWW59C99v6uxiln3+/GQg+hpULu2PBqDWKAHRA0tI\nk2uWgGjqppsGWp8HMM+l5NJNnmaDdYVUBBQZeYMzAG0mAd0J4NX8A4QQGcAXALwGwG4AbyOE7CaE\nXEQI+XbRP4NNXXWHoykSPv/2y3Dxhm7P4++5ZgsIAfaNd/v+3nU7B/DGS8dYOmAl6Ekm5RTEbHIG\n4Jx0UkEzusm0ZMAOBJ+eT+OHh89jc18Y2wej0BQJqkw8HsAjx+cwk8zhvdduwaa+iKe2QDftXjoR\nn03o9jfswf961/6yU5BGu0N46Mg0XjiXKNmkJYngZVv68PBLs5h2KmYHOQPQFw24QWCjqBCMNk3j\nZilXY7OTCcRj1wFIbINlnkVaR0BxH4+HFCxljLIeAN3AaAuEpYw3BgDYBmDJqeEA7BgA3/SPN9yD\nsSAUiThpoN4YAA/1AHJ6gf09yxVh+UHXHdbkmnPxAXszpvbiKGuZrSOZM7BzyD7o+NUC0CycC4dt\n768vqkGVliMB2Vug31xqv9+h87TrzQKiBl2VSWnciQsCU0PUVhKQZVkPASgul7sSwFHnZJ8HcDeA\nN1iW9YxlWa8r+meqyetel1y5pRePfPxGXLW1z/fnO4di+Lu3XAK1TJ0BT5STFQZipR5AVjcR4tzQ\njb129svPj0zjpt1DbKMunj9Lv8T7xruwqS+Mk3NuyiR9Hp8FRBnrDuFlZd4XAHz0lguQzpt4aTrl\nu0lfva0PJ2bTeGpiEYS4GzzgSEDJvKfff3dRDMCtL6ieQbVzyG6lzG82tBLYLwuo21OBrSKR4yUg\nrwdArzmXste7lDVKvKItTiD6xGwKulnAVCKH0aL2GZSusIqwJjseQKFEAqLwJ+HMMgwA1bcjAaUu\nD4A/3VMPgKaA7nSylPxSQenrXThiP2fAkYDKBaCPTiXwR/ccZNlegO1ZMgmoiuEojgHUG+imBn0o\nHnQNQNrt+MskIL0NJaAyjAHgh7VOOI/5QgjpI4T8E4BLCSEfr/C82wghBwghB6ana3dB1wu04Vej\n8M3j+qMBRAMK+qMaO7Fn9CIJyBlOX7CAm3YPs8cjAcVTlHR0KomesIq+aACb+8LI6gWWakplET8P\noBo3XDiIB//4evzZay/E771ia8nPr9lmG48fPHsOfZGAp9iuP6ohbxawlDHYybw7XBwDcCewVWPf\neBdyRsFTfUyzgNwYgBtc7uGMSjxoewC08d1sOQkolUc6bzdx8/MAAFszn0rkYFnACJc9xf/dukIq\n+xtli/6mPLwEtBwPgM5DCGlyXWmgdHPnGw7SFFBqAPzqJqgEs2/M9oZHu0MVg8C/ODKDbz4xiTML\nGU9FNAsCV/EA6Mk8XqYS+JtPTHi83WKotzbSFWTeBn1frBWEbratBNQwlmXNWpb1fsuytlmW9akK\nz7vDsqz9lmXtHxgYWK3lrTtoyijg6uVjPe6M3Uze9Jwk6YzknrCKyzf1sMdDmuzpBUT73ti/45xU\nnYAl6wTq4wHUQiSg4LbrtuHqbaWewgVDMfSEVSRyhkf+Adxq4JlUDlm94EhXEgKK5PbNz+QRDShl\nq7R5LnGkuae5FgK60wyuWAIqLi6LBe3md5PO4B6+UZ19HUcCSueZRlwcAxjvCUGRCI7PpFiwdLiM\nB9Adcj0A2ojPD7rhZHWTGerpZD0egCsB1XM6ppvgzqEYTsykYZgFlgK60ylU86sFYBLQSAzf/MA1\nuHn3kGMA/F+bTkhbSOusJ1JIldlJu2oMwPl9VgfAeTkT82n80T1P4WsHTpX9/fl0HrJEMBALMBmS\ntmOPBRUuCNyGElAZJgFs4P5/3HlMsAbgM3HoBtkX0Viwiq8DAOxNhxDglRcOsdRCwJ4uxqeBvjSd\nZEHqzSyuYJ+MGvEAqiFJhBmG4hoJZgASOc8pOKTJLCi8WLRRV2JjbxjdYRVPnXbbCNM0UE22K23p\ne6Uzmylxp7V33ixg52AMlgXMc03b6GjJ2VSeNY0r9gAUWcLGvrBtAJzNkg8C+3oAZbKAKO5G2GgM\nQKkrQ4bq+5ds6EbeLOD0fAZnFzKQCLDNKXrzDQI790mVJFy2sQeKc+/LvTaf7eVKQDLLPqs1Cyjq\n0wriR87ciFSFrrhzKdsTDKpuxXWSHYjsWIRuuhLlWvAAHgOwgxCyhRCiAXgrgG81Y1GEkFsJIXcs\nLtaehiaoD4XLpacbZk9YcxtWFRmAsKbg82+7DH90807PdUKaGwOYT+Uxm8ozAzDaHYIsERYwZR6A\ntjwPoBpXO1lTJR5AzN6Ap51+RkFns+OHpxf3AaoEIQQXj3fjqQnOAJgFaAoBIQRhLi5ij+z0egAU\nOh/A083ScIPArgdQer+29kdwZCrJ5vrygX9VJsxIdzsxAFsCKpTNAgoqpUHgZM7wHXDvB90QbQ+g\nfgno0o22V/XSVBJnFrMYjAWZ4fSLAVAPQJbdw4gik/IGgBrkIgMQ5AxfJZgH4CMB/cjJjKs0T3o+\nlUdvRHUa9TlpoI7xiwbcWAQ1iG0VAyCE/AeAhwFcQAiZIIS8z7IsA8AHAfwAwHMA7rEs61AzFmVZ\n1v2WZd3W1dVV/cmCZUM/zLRqtjeiYi5lSxK6aZXoxb+5b6SkVUOEMwBHnSAelYBUWcI4lzLJPIDA\nypxurt7q7wHQFglU/2UeADc8nW8FXQsXb+jGi+cT7D3pTisIwK7EzTrDZujMZgp/mqdZXnwgmJ6k\nbQ9AL/kdypb+CI5OJfEvvziOse6Q5zmEuPUIXSENEc2OO+TNQvkYgMoHgblGczV6AXTjjWhKXSmS\n9HR/yQZbVjw6nbRnG3QH7bkRxD8LSGcegGsAVFliBrQY3gPI+sUA6vUAHAloMaPj18dmAVRuiz7n\neIJBxTUAqZwBQmzjSz879H6slgRU01HMsqy3lXn8uwC+29QVCVaNWEDBdCLHTsg9EQ0Z3WSSRLnN\ngiccUJByJJ6XaKM6rk6BTwVN5VfWA9g2EMGf3HIBq5egdIVUxIIKJp3h9tSzCaqymwWU0ZnhqoWL\nx7tQsOwupFdu6WUSEGBvEjTgnDcLRR6Awl3DMQCpUg8gkTVYgLg4BgAAv3PVJoQ1BZds7MYVm3tL\nfh5UZSRzdqvhcEBhr1E+BlAqAQG2ARjtDuFzPzmK9167uWyWFDUAIacddq3Q0/1odxADsQA+++Mj\nyOomXnPRCAghnsFFPLQXEB+zsesAysQAnMK8RW54TEiVkVbs91pVAjLcecqKRJgH8LMXp2EULGiy\n5DGcxSyk89jaH0VQlVhNQSpvIqzaw5Ho/ade32pJQCvzTRSsCaJBBRFNZhsybZBGe8sEa9Dqw9wm\nenQqiYAiebyETb1hPHlq3m7AlVtZD4AQgv9yw3bfn411hzC5kEHOcGWQkCZ7crKLC8wqQWsxnjq9\ngCu39Np1AM6XeJOjz8+zTqDudelm3hvR2EAfvrEcL2FQw+mXmbSpL4I/vGlnyeMUutF3h1VENJll\noZRPA+WDwF4DcODEPD774yMYiAXwzqs2+f4+3UAjmlx3Gightufw/t/YhsdPzmFzXwRvvMxOKIwF\nVd/22azwjJOANJmUzc+nrTnsCVx2vEBxEgGA2oPAQUWGIrvFbj88fB59EQ2bnCaB5ZhL6bh8kx0D\nMAsWDLPgVMXbf1sqAfEjIleDtjQAhJBbAdy6fbv/l1nQHGJFTeVogzQ6mrEWD4BPA31pOomtA1HP\n3IFNfWEksvZ0qpX2ACox3hPCxHwGsaDC9G4+BlBPEBiwZaax7hAOTiw483At5sbvGIzi4ZdmMZek\nxWXuqZl6AHYbaxWKRDzFYDnOABx3YicxHwmoGq4EZAeB+VOvH7wHwM9ImE7mWDbQ4aLRmzx55gEo\nMAsWCgXL8zkoRyJrIBpQIEkE73v5Frzv5Vs8P485bcuLoRuwIvExgAoSEBeTCWsyOwRoSuUYwK+O\nzsAoWOygEFAlT8HZr47O4DcuGMB0Ild2eA1tBGfHAJzCM6OAZM4sMQBMAmqnGMBqI2IAq8Or947g\njZeNs/+nxVOTTiFOLQYgxM3APcplAFFogdnEfIZ5AOVkiJVkrDuEyfmMp801lYCyTv61n9RSiQuH\nYzg2nWJ6NP0S7xiMIWcU8PSkHaD1eABBd46BJBH0RjRPDIBvq3xyNsVmAdRLUJPZhDI+66p8JbC7\nEaYc6UgitgdA6x0OnymflEHXTa+v11gpm8garCrdj1jQXwKiEowieSWg8mmg3iwg+tmulgX06Qde\nwCfvexY53QQhtqFUZLfn0FJWx1A8aB8myngAS1l79GpPWPN6Wtx0PPr4EosBCAlIsMIUu/M064J5\nAFr1jSfi5H3TBme/zRkUwM3ImUpkkcqbLAd/tRnrCSGRMzC1lMOmXju9kBqvxaIZA7XSFVbx/LkE\nO/2qjhyx3WlhcODEvHNdbyEYACaT9XGN6gCvBHRiJu2bAVQLIVVCV0izs5K4DbacrEeloZxuVwLT\nzJTpRI41aXv+XAKGWfCtldDNAmSJsBOuYVqopdwjkdUrejjRgOJbj0A9ALVIAsqbdkfT4pYizAPI\n6J6eTYosQZFIWQloOpHDxHwGEwsZBBQJhNgT/YxCgSVLBBWZ1Vr4QauAeyPuvIKsbiKZM5g37AaB\nV1cCaksPQNAaeoskoHJ6MQ9tf3zw1AIsyz798gw5lcvnl3IlswBWkzGny+hUIsfeV0iVkMmXThmr\nla6QXdRFZQf6JaZe0KNOj3veA+iJaNg33oVrnJRVu1Gdd6YtlTWSOcM3A6gW7AZ69t+Gv+e1VgJH\nAjIGogFMJXJ4cSqBeFBBzih4upDy0JnI9EReaypoImt4AuPFxIKqvwRkWpAl4tno6cHCb8gOn+5b\nXOSoOa2Yi7Esi6Xo/uroLPvc2H17LBbMDWl2G/ByBmCB+3yxiXFOum20nAQkDIBgtekKqSCkPgNA\nN5f7nzoDiQBXbfVmpNAirKlE1ncWwGrBzxkorgOodRZAMfGgimTObS9Bg8DxoIrheJBVVfNDZlRZ\nwrc++HLccKHdH7EvopV4APSeAf4ZQLXw/t/Yho+9ZhcAb8ylnAGgxitnFJxpWXZ86LmzS1hI6/jN\nfSMAyscB8k4zPHoPaq0Gtmdf1y8B6YWCpyARQMXXZpXZaZ3NA6YEFP8uokmndgKwp6vR2JHiBJtp\nXCCo2h5AuSwgasBiQdUjAaVybmW2awD0inPDm01bGgBRCNYaZImgO6TWFQOg8sL3nz2Hyzf1sBGM\nFE2R0BfRXA9ghTKAqjHeU1opGyyWgGoYpclDDQbNsOGlrR2ODBTmcs394DuVAvZGGg7ITCqqpTeR\nH1du6cVNu+10WP6el4sBEEKcoSR2HUBYlTEQC7BK41fvHYGmSDhUJg5A6yBoXn6tHkAya1SWgIIK\nEj7BVdO0PDUAgBsQ9tvM+XTfdFGRY0Dxn4dcPPaTHhzsNFC3apcZAKf2oxgq68SCCld4ZiKVN9hM\nDZYFlDUQXKXTP9CmBkAEgVtHT8TtnV+TAeB6z9MNp5iBWADTTgygVR5AX0RzT/40DVS1WxcfceYs\n03qIWqGnc6pR8247lYGqzRjui2qsTw/gbqRUjluuB8DD3/NKXp09FrLATsh8htiukRguHI7h8Fl/\nD0A37F5I9ORaazFYImt4OtMWEw+qnjbJFKNglZySNaW8/EQ364xuYjGd9xhCrYwHQL8He5yBQ/Te\n0aZzOa42IKjKKFj+2UQJruKXl4BSXBZQgPMAAjV875pFWxoAQevo5QKW5doG8IS50yXfJZRnKB7E\nVCLnyXpYbQghbN6wGwOw/33Xr07gis09rGK4VqgHMONUy3o8ACcWUq29dH/EO7SeFpTRlNx6ZSk/\n+Hte6W9qD4a36wBCmswqxLvDKgaiAewZjePQmSXfU65dB0FYULauLKAqEhB9XsnryUUSkOxvACzL\nQkY3WSzm3FLWYxTLzUOmVdCv2Wt/runnhtYBuLUBblDZLxOIrj0edGMAmbzjAbAsIDcGsFr6PyAM\ngKCIHq6Pfi0xAPpF2j4YZW2KixmMBXB+qbUeAOBm3jADQEcnLuXw7ms21309ZgCS5SWgaplFfVH7\nftOqX920mGwG+LeBqJdaYgCAnQpq9wIyPB7AzsEYCCHYPRLHQlrHw8dmSzZMFgMoswn7kXUqpSu9\nRxokLQ4E0yAwj1rG+8gZBRQsYNgx8MU9kej7LoZ6ALfsoQaASkC2B8C3lWZzIHzaQdC1RwJu76HF\njA7LglsHILtzJIQBELQMjwdQRxC4nPwD2B7ATDKPRFZvWRYQ4MYBWAzA+fdwPMi+5PVAUzTpRsGf\nSLc7bSV6qngANGZCi8HyzsmW/t5y00B5+NkPFQ2AInMSkMKC0dSYXea0AX/7Fx/BJbf/kLX5BtyB\nOOU2YT8SLDhaOQuIfy7FKFieGgDAvf/Fcg4N1o6UmZqmlRklOZ3IQSLA1oEodg5FmcFXnToAPghM\ns+H8AsGJrF18psgSCyTTuFGkKAgMrF4NANCmBkAEgVsH9QBkiZS42H5s6Y/gd6/dXLZFAAAMxgMw\nCxbOLmaXPQugGVAPgNY30E3gHS/buKzaBLoh0M2b/xL3RDRs6Y+wOQrVrkED0VQC6m2qB+DKDJWq\nc6kUQkdH0hkDFzp9+feMduHBP74et79+DzK66emGSkdiKmU2YT/44Gg5XAnI2w7CKNQuAdGTOj8z\nwZsFVC4InENvJABZIrjjnfvxyVv3ALA9AKNQ8IyWpLGwdN4OBH/54RNMQkrmDGaE6aFjxsn8Kq4E\nBlavChhoUwMggsCtg0oPQafopRqKLOEvb93D9HU/BmP2F88sWC32AOzNmG78l27sxk27h/COCsar\nEnRzphKQVmRE7vvgtfiDV+2oeA3WGsDZgFYiCEw3mWoxnYAiIZE1YBQshDUZW/oj+PzbL8VvX+4W\n923uj+CtV26ARNzmf/a6LagyYfegFg8gyYKj1SWg4kygShIQX00NuOM5R+L+Q3MCavkYAJXBNvdH\n2AHCbjvt1gEEVW4QUN7E+aUcPnnfIdx30B6Pwsc56N+btgkJaz4GYBUlIFEJLPBAPYBaAsC1Mhh3\ns0laGQPY6gwYofLKeE8YX3zX/mVfL6zZnSFdCcj7xa3l9M768DsbEB0t2dNEDyCgSJCIm7FVdi2q\n7GaAOX+n1+0b9bmejI29Ybw0zUlAzrppKmYtYyFrkYDinATEVyFTj4OnnARET+oeD6BIAvLL3plO\n5oWIxoEAABhBSURBVNEfLZXwVKcSmA4TCihub6G0brKhSrS+JJEzEHXeB5V3aMynOAjMP2c1aEsP\nQNA6eiP2B7WWAHCtDHEnr1ZlAQF2B897/8u1uHJLafvk5UAIQVdIZZvmsnr2cGmBgDtacpPTQ4kf\n9NLIOiOaUrW7a0CRWAfTav2atg1E2RB3gJeAahuyDtQmAdEU0Z+9OI3L/+pH+Npj9thFs2B5OoEC\n4LwP/xjAYDwI6tR6PQDZtxfQDOcB8NA6ANoimg8C85Xl1BAksjqr56AbvWsAFHZNujYRBBa0DHo6\nriUAXCsD0fbwAAB79GAt0latxEOqbyFYrdAvO92kaAzgyi29+PlHb6hrRkElwgG56t80oMisb01V\nAzAYxbGZFGu7QKWreiSgJS49shzUONz/1BksZnScnrOrq3WfILDrHViYSmTx4bufRCpnMAkoosns\ntUIlaaClqaPTyZzns0uhdQCeSmDVvl6aMwDMA8i6MQBJItAUicWNaCEYIa58tu5jAILW0bsCEpCm\nuJp2Kz2AlSAeUlmr5eIYQC1IEvFIEFRKIYSwTqrNIKIp1Q2AKrEma9UM9baBCPJGAZPzzobMCsFq\nrwSuRQKiAfGLxrqgOs3eAPuUr5TEANzXPnBiHvcdPINDZ5Y86Zo0LdcjAfkYgETOQN7wtuWg0DoA\nOmQmqPAxAKPEA0gW1ToEFcnNAgp41wEICUhkAbUQqj03UwIC3K6grfYAmg3fqmE5EhBgb7z0NGmf\npJvnoVBizlS0iuvg1l+LBASAyUB0IA4LxNYwFIbmx0erZIZ950Mvx/9+/9UIclKNYZZKQConP9EZ\nFfPpvDsCUpVZ1lVxL6DiIDDN4PGXgCQmAamy3beHGQDdZENd5pkHoHsC3UFuZnSkyBPh/70atKUB\nEFlArSMWUKBIpKkSEGDrr4D3A98J8JW6taTN+kHz7wF4Rks2k796w17WHK4cvNGv5gEWGwBav8BO\n4VVGLAL2xhhS5aqNz0a6Qgiqsqdpm50GWr4VhNv8Lc8225DmGoBgcRpo0XppdbefB6A6w+czebOk\nqtwrAeVhFiyk8qbXA+DucyTAG4DSgPBK01nfRkHDEELQE9GabwCoB9BhEpDHACzzixtU7SZsgJsF\n1GwuGq9+mKrHA+iJaOiNaB4PwFMIVkMriGptIIqx2zBTA2D5SEBu/CGVs+/nfFpn0lxYVVhrjpJe\nQIZ3jgDt7+TrATgSUM5wDYDszPXN5E1mfObTeZbq6jUAkrNe4vlbMwloFXsBCQMgKOHVe4ZZ9Wez\nGHJSQTvNA+Dz9JcTAwDsjTdrmGy0ZCsG5tjr4LqGqtX/Tlv7I3hpyk4Fpet2YwA1VAJXaQVdjKq4\nBkA3LcjFQWCuGyhtrjefzrPAb1CT2IwE/v1Rw5c3C+weuB5AaRooawWRN9lmDoANhaEzjLN6gWWI\n+XkAxXIoCwILD0DQSv7bb+1t+jWHHQmoUufHtYhXAlquB2BXolJ5YyU8gFrgN55akgC2DUTxo+fO\nA3ClK7WOgTDJnMny42vB4wH4NIPjJSAq+yykdKiSXQehyRJr+R0qigEAds8gagCmkznIEvFt5eG2\ngih4POWwMxSGSkAAcHouDcA715nWfhTHPrQWxAA669soaFvecOkYQprCqik7BXq6lCVSUplaK0FV\nRtYw2al5ubGERuHTD2vJ1to2GMHXDuQxn8qz7CUqg9WSBprVzbp632uKO4zd9GkH7VYCez2AaFBB\nWFNACMGmvjBiAcVzImceABcHmEnk0RfRfFtn0JGQWcM7VyDoBPP9DAC/2dP7XCyztSILSBgAwaoQ\nD6p40+Xj1Z+4xuAbhC2XgDOSsHi05Grj2cxq2IS29Nsy4YnZFMteqjSUpZicUair3TXft18vFEoG\nwtC/gVFwYwALaR39MXejfuNl43jVrqGSgTB0PZTppH8RGACokt0Kgg8CA9QDsNNA6ajP006arJ8E\nFCn2AEQdgI1IAxWsFWi3zkY2bdcDcIbLt1gCCqlyxaZxFOrNnZpLw7LgaQddiweQ0+trfczXS1Tq\nBZTnJKD5dN6ZAWz/TJaIp+U54G64Oa6V80wy55sBBLgFZ6m8UZI5lc7baaCb+uy2I6dmqQRU6nEU\ne1l0HSINVKSBCtYI9ATbiG4fUCRk9QLb3FodBK6WAUShc5ZPzNibnKpIkCUCidQWA8gbhfoMQFEQ\nuLwEZHESkD0EvlJQW+MMB2U6UckA2IYnWTS+kY6FXEjr2OR0gT097xMDoB5A2SDwOi8EEwjWCq4E\n1JgHkOM8gNU8AfIwD6BGA9AVUhELKDgxa2cC0XugyFJNE8H4oGst8EFg06cdNG98WBA4nbdnAFd4\nT64HYF/bsizMJvMVJCB3ehd/r8KajNlkHkbBwqZexwPwiQHQzKESCUgUggkEawsaBG7EA7CDhwUu\nCNwiA0A3pjpSdcd6QswA0ApmTZZqk4AMsy69m48BGGZpLyDA7dNDPQCjYGE6kUOowusUxwCWMgby\nZsE3BRRwPYBEzvDESoKqjPNLWQB2B9yQKiORNexOrFppfKVsEHi9xwAEgrUC1XYb2bTtgSQmO922\nygDQjamePlCj3SE2Gcz1AEhNElBOr18C0rkgcHErCMA2PrpTCEZDBGcWMhVbkGhFWUDTSXsTL+cB\nsI6nRqHEA6C9lLpCKptBHA0ongaEVAIqTgOl96KWAHyzEAZAIGgARZYQDSiNGQBVQtZw6wBanQZa\nawwAsAPBtOcNvQeqswlXI2cU6uo55ZWASiuBAdf4ZHSTtSFfzOhVRmHSOgBbNppO2I3a/DqBAvBk\nHwU8hWDuht4dUlnVcayo1iHI7nOxAZBLrrnSCAMgEDRIV0htTAJS7CZndANqXSFYfUFgwA0EA272\nkp0mWdkDKBQsp/K2/iAwrZj26yFEJaBUzvDUnFQyNPR9ux5A+TYQADyvy5/WeSMTD6nocWZrFFc7\nu2mgra8DEAZAIGiQeEhtqIMnPfHR7pitqgNwg8C1xwD4UaD0HqiKVHUiWJ4FvOvwABwDQGcQFNcB\nALYByOomckbBs7ZKRk1jHoC9pkqN4OzXcF+3WAKidHk8gKKTfrU6AFEJLBCsHS4cjvnKEbVCT5G0\nP37L00DrkGX4UzbdSBWnUKoSNOOm7joAs8B0dtkvBqBIrBKX904qxTVKJKBkDqpMyhap8cFnPg2U\nf42usBsDKJGAWB1AuSygdV4JTAi5FcCt27dvb/VSBIKq/P1bLmno95kH4HSObJUExLTpOjq2jvMS\nkCcGUNkDoJttvVlAullwC+Z8s4AIMwCj3AzgyhKQNwg8k8ihLxIoWwynlPEAqAQkESCqKayPUHGw\n160DEFlAvohCMMF6wvUAvMHU1WY5MYCBaIBJIrwBMHwGwnzvmbP4zA9eAODOQK63DsCyXKnGLwtI\nkSQsZOisYZUN7KlHAppO5tAf808BBbwSUHErCMCWBCWJlJeAFP8gcG9YgyqTEs9gJWlLAyAQrCfo\nJpJocQwgqElQJMI6ZtaCJBGMdNleQLU00O8+ew53P3YaAOcB1BkEBoC00+fHNwisSGwiV1iTWduH\nyllA3jqAmTKzgNlr8EFgtTQGQKUjlgZaZAAu3tCNV+zox66RmOfxN1w6iu9+6BVVJ6Q1E2EABIIW\nQzdBOiRdVVqUBqrI+I/brsJbrtxQ1+/ROIBWRQJayuhsPCPdbJdjAFJOkZdf3EWTCRvGHtbcATCV\nDIAqExDCB4HzZQPA9uv6G4BQiQGwX7t46P1QPIh/e9/L2NooAUXGjiGvUVhp2jIGIBCsJ1wPwN64\nWuUBAMAVm3vr/h0abKWGS5UJk3h4FjM60nkDlmVxMYD6soAAsCpfPwPAy0/hgMxO4ZWCwIQQp9Gc\niULBsj2AMimg9mtwEhAfBFa9BqCbKwRrV4QHIBC0GBr0Y1lALQoCLxeabumJAfh5AFkdBctOAV1O\nFhC9Pu3z4xcr4WWhsCazU3i1Eae0JfdiRodRsCp7ANxr+KWB0ilxo90hKBLxpKO2G+1rmgSCdUJx\nELiVHsBy2DkUhSy5aZP2yMTSIPBSxjZw2XxhWRIQfS7t9e83gIevx4hoCjuFV2tvEVBl5M1C1SIw\nwOt5VJKAhuJB/OyjN2AkHkS7IgyAQNBigsUewBozAK/dO4KLPtLFTs1qmSAwnZWb1g0uCFxfFhAA\nZHR6n/wlIIrHA6hiADTZ9gCqFYEVv0bxSEjAOya03Sfgra1PmkDQgdBNMJkzGhot2SokibABKIB/\nGmiWa3aXyZvMAwjWWQcAuB5AuW6glLCmuDGAahKQascAXA+gfCYUn37K5+xHAwq6Qiq2cPei3REe\ngEDQYngPoFWN4JqJIhPPfF3APf0DQEY33RjAMoLAmTxNAy3vARBi39eLxrsx1h3CaFflkzhtNDft\neAAD0fKyjVomC0hTJPz8T2+oq512q1k7KxUIOhS6CSZzBitcWstoztB0Hqr/A9QDWEYdADeKESjn\nAdhGIeIMgb9kQzd++bFXVr12QJWRM+wYgCZLbNSnH55K4CIDVpzy2e4ICUggaDH8JtiqNhDNxC4E\n80pAJR7AMoLAquLNAqrkAdQz04CuI2eYOLuQRX9U8/TvL4a+riyRNRevKWbtHzcEgjUOvwmu9Q0F\noFlAxR4AZwDyvAGoPwhM6wAqBYGL++xUI6BImE7kcGhyCTftHqr4XCoBVYsrrAWEARAIWgwhxDmB\nFjrCAGhK6UhIWuUM2B5AVjdBSH3DbwLFHoCfBOQUo1WaAFbu2s+fSwAA3nrlxorPpR5APQHsdqUt\n3wEh5FZCyB2Li4utXopAsCrQYGJHSEA+A2EWfTyAgCJVlFqKKe4F5F8HUP9UM8D1RLb2R3DF5p6K\nz6VGup5pZu1KW37aRDdQwXqDniY7wQOgaaCW5XoBHglIN5HTzbr73hf3AvKtBJZoS+v6PAB67bdc\nsaGqUaKFYMIACASCpkA3w0Ymi7ULLF3TafwG2EFgemBPcx5AXdeVawgCKzQLqL7NOaTJUCSC3758\nvOpzZUlIQAKBoInQzaQTJKBLN3QDAL799Fn22FLGQG9Eg0TARjbWO/ikOAvIbyCMtswsoPe9fAv+\n+Z2XV6wAphBCoMqkI4LAa//TJhB0ANQD6AQJ6OptfbhwOIYv/eI4k4GWsjriIRUhVWZ1AHVLQLQV\nhCMB+Y2EdLOA6pOAtg1EceOuytk/PIokCQlIIBA0h06KARBC8N5rt+D5cwk8fGwWgB0DiAdVhDQF\naacSeLkSUIp5AD4TwRyjUM9Yy+WgyEQYAIFA0BzoZtIJBgAAXn/JKPoiGr70i+MA7DTQeEhFSJOQ\ndWIA9W6gkmRLL2lndrLvRDCaBaSubIa7KgsPQCAQNAl6Gq73VNyuBFUZb9o/jp88P4WsbiKR0REP\nKgipshMENpf1XjVZQlovHwSmXkJkhT2ASEDuiLYda/8dCAQdQIB5AGs/C4iyb6wbBQs4OpV0YwCa\nwlpBLGdSlqZITAIqNxEMqL8QrF7+8R2XV5wZsFYQBkAgaAOCHRQEpuwYigIAjkwlsJQx7BiAKrFu\noPUGgQHv/anUDK7eQrB62TvWGTVKnfNpEwjWMIEOSgOlbO6LQJEInplYQt4soKs4C2gZefSap29S\nJQ9g7evzq0HnfNoEgjVMJ3oAmiJhS38Ej5+cAwDEQwrCjgSUXUYWEL0mYBdj+VXs0p9H2ngQezvR\nOZ82gWAN04keAGDLQIfOLAGwe+UHG6gDANwgb7mpaZdv6sEfvmon9lfp5yOw6axPm0CwRnE9gM4J\nAgPAjsEYGw9J00BpEHg5HgD9Hb8aAMDOPvrwq3Ysy7isR4SfJBC0AawVhNxZG9fOoRj773jQkYDy\nJvJm/a0gAFci86sBENSPMAACQRvATrZKh3kATiYQYHsAQVVmTeKWJQHR+9RhnlKrEGZUIGgD2DyA\nDjvZ0kwgAE4aqLvpNxoEFjROZ33aBII1SqcGgTVFwub+CAAgFlQ86ZnLrQQG/GsABPWzaneREPJb\nhJAvEkK+Rgi5ebVeVyBYC3RiGihl51AUAcXuncN7AMvppSMkoOZS06eNEPIlQsgUIeTZosdfTQh5\ngRBylBDysUrXsCzrXsuyfg/A+wG8ZflLFgg6j05rBsfztis34rbrtgIAgrwH0EAhmAgCN4dag8B3\nAvg8gC/TBwghMoAvALgJwASAxwgh3wIgA/hU0e+/17KsKee//9z5PYFA4BDo4JPtK3YM4BU7BgAA\nYU8MYPl1AH59gAT1U5MBsCzrIULI5qKHrwRw1LKsYwBACLkbwBssy/oUgNcVX4PYZXt/A+B7lmU9\n0ciiBYJOgzaD65RuoOUINRoDYB6AMADNoJFP2xiA09z/TziPleP3AbwKwJsIIe8v9yRCyG2EkAOE\nkAPT09MNLE8gWDts6Y9g51DUkzffiXgNQCMeQGcbytVi1eoALMv6LIDP1vC8OwDcAQD79++3Vnpd\nAkE70BvR8MAf/karl7HieNJAG4kBCAmoKTRiRicBbOD+f9x5TCAQCHxpVh2AkICaQyMG4DEAOwgh\nWwghGoC3AvhWc5YlEAg6kXCjEhALlgsJqBnUmgb6HwAeBnABIWSCEPI+y7IMAB8E8AMAzwG4x7Ks\nQ81YFCHkVkLIHYuLi824nEAgaBOCTSsEEx5AM6g1C+htZR7/LoDvNnVF9nXvB3D//v37f6/Z1xYI\nBK2jWTEAWQSBm4K4iwKBYNVQZYnVOjSSBdSJ9RKtoC0NgJCABILOJdhAzYOoBG4ubXkXLcu637Ks\n27q6OmPwskAgcAk1wQCUGwgjqI+2NAACgaBzCWsyFIks6xRfbSSkoD6EARAIBKtKUJWX3fJCFRJQ\nUxF3USAQrCohTWa9j+olIILATaUtDYAIAgsEnUtYW74H4LaCaMuta83RlndRBIEFgs4l1IAEJFpB\nNBcxFF4gEKwqL9/ejw294WX9rmgG11yEARAIBKvKe67dsuzfZa0gRBC4KYi7KBAI1gy0CZyoA2gO\nbWkARBBYIBD4QWMHsogBNIW2NAAiCCwQCPyg/YNUkQXUFMRdFAgEa4Z4SMFHbtqJW/YMt3opHYEI\nAgsEgjUDIQS/f+OOVi+jYxAegEAgEKxThAEQCASCdUpbGgCRBSQQCAQrT1saAJEFJBAIBCtPWxoA\ngUAgEKw8wgAIBALBOkUYAIFAIFinCAMgEAgE6xRiWVar11AWQsg0gJPL/PV+ADNNXM5aQLzn9cF6\ne8/r7f0Cjb3nTZZlDdTyxLY2AI1ACDlgWdb+Vq9jNRHveX2w3t7zenu/wOq9ZyEBCQQCwTpFGACB\nQCBYp3SyAbij1QtoAeI9rw/W23teb+8XWKX33LExAIFAIBBUppM9AIFAIBBUoOMMACHk1YSQFwgh\nRwkhH2v1elYDQsiXCCFThJBnW72W1YAQsoEQ8lNCyGFCyCFCyIdbvaaVhhASJIQ8Sgh5ynnPt7d6\nTasF+f/bt58XG6M4juPvT9MoDbKRJleNhWwsqGk2IwtFfkxYUqyUDUUWyj8hfwAbkUkNJYpGpjTl\nx5gxiJGkKTOpWUjMSvhY3LOYjZqFc0+d5/uqp/ucuzmf29O93/N8n3OlLkkvJd0tnaUTJM1KeiNp\nWtKLrHPV1AKS1AV8AHYDc8AEcNT2u6LBMpO0E1gErtreWjpPbpJ6gV7bU5JWA5PA4ZqvsyQBPbYX\nJXUD48AZ208LR8tO0jmgH1hje6h0ntwkzQL9trP/96G2O4AB4KPtT7Z/AsPAocKZsrP9GPhaOken\n2P5ieyqd/wBmgA1lU+XltsU07E5HPau3f5DUAg4Al0tnqVFtBWAD8HnJeI7KfxiaTlIfsB14VjZJ\nfqkVMg0sAKO2q//MwCXgPPCndJAOMvBQ0qSkkzknqq0AhAaRtAoYAc7a/l46T262f9veBrSAAUlV\nt/skDQELtidLZ+mwHek67wNOpRZvFrUVgHlg45JxK70XKpP64CPAddu3SufpJNvfgDFgb+ksmQ0C\nB1NPfBjYJela2Uj52Z5PrwvAbdqt7SxqKwATwGZJmyStAI4AdwpnCv9ZeiB6BZixfbF0nk6QtE7S\n2nS+kvZGh/dlU+Vl+4Ltlu0+2t/lR7aPFY6VlaSetLEBST3AHiDb7r6qCoDtX8Bp4AHtB4M3bb8t\nmyo/STeAJ8AWSXOSTpTOlNkgcJz2inA6HftLh8qsFxiT9Jr2QmfUdiO2RTbMemBc0ivgOXDP9v1c\nk1W1DTSEEMLyVXUHEEIIYfmiAIQQQkNFAQghhIaKAhBCCA0VBSCEEBoqCkAIITRUFIAQQmioKAAh\nhNBQfwGGqd9/90rFTwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x103f3abe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_axis = np.linspace(0, 5, len(losses), endpoint=True)\n",
    "plt.semilogy(x_axis, losses, label='rho=0.99')\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到使用 adadelta 跑 5 次能够得到更小的 loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**小练习：思考一下为什么 Adadelta 没有学习率这个参数，它是被什么代替了**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当然 pytorch 也内置了 adadelta 的方法，非常简单，只需要调用 `torch.optim.Adadelta()` 就可以了，下面是例子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, Train Loss: 0.356505\n",
      "epoch: 1, Train Loss: 0.158333\n",
      "epoch: 2, Train Loss: 0.120510\n",
      "epoch: 3, Train Loss: 0.100807\n",
      "epoch: 4, Train Loss: 0.084741\n",
      "使用时间: 47.90947 s\n"
     ]
    }
   ],
   "source": [
    "train_data = DataLoader(train_set, batch_size=64, shuffle=True)\n",
    "# 使用 Sequential 定义 3 层神经网络\n",
    "net = nn.Sequential(\n",
    "    nn.Linear(784, 200),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(200, 10),\n",
    ")\n",
    "\n",
    "optimizer = torch.optim.Adadelta(net.parameters(), rho=0.9)\n",
    "\n",
    "# 开始训练\n",
    "start = time.time() # 记时开始\n",
    "for e in range(5):\n",
    "    train_loss = 0\n",
    "    for im, label in train_data:\n",
    "        im = Variable(im)\n",
    "        label = Variable(label)\n",
    "        # 前向传播\n",
    "        out = net(im)\n",
    "        loss = criterion(out, label)\n",
    "        # 反向传播\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        # 记录误差\n",
    "        train_loss += loss.data[0]\n",
    "    print('epoch: {}, Train Loss: {:.6f}'\n",
    "          .format(e, train_loss / len(train_data)))\n",
    "end = time.time() # 计时结束\n",
    "print('使用时间: {:.5f} s'.format(end - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**小练习：看看 pytorch 中的 adadelta，里面是有学习率这个参数，但是前面我们讲过 adadelta 不用设置学习率，看看这个学习率到底是干嘛的**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mx",
   "language": "python",
   "name": "mx"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
